package com.bfmet.crawler;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.BulletList;
import org.htmlparser.tags.Div;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;

import com.bfmet.entity.News;

public class NewsSpider extends Spider {

	@Override
	public void run() {
		try {
			this.spideIndexPage("http://www.cnmn.com.cn");
		} catch (Exception ex) {
			this.logException(ex);
		}
	}

	public void spideTwoLevelPage(String url) {
		try {
			log("Fetch two level begin..." + url);
			News news = new News();
			Parser p = new Parser(url);
			NodeList nodeList = p.parse(new AndFilter(new TagNameFilter("div"),
					new HasAttributeFilter("id", "newsContent")));
			if (nodeList.elementAt(0) instanceof Div) {
				Div div = (Div) nodeList.elementAt(0);
				String title = div.getFirstChild().getNextSibling()
						.toPlainTextString();
				this.log("title:" + title);
				news.setName(title);
				p.setInputHTML(div.toHtml());
				NodeList nl = p.parse(new AndFilter(new TagNameFilter("div"),
						new HasAttributeFilter("id", "content")));
				if (nl.elementAt(0) instanceof Div) {
					Div divContent = (Div) nl.elementAt(0);
					String content = divContent.toPlainTextString();
					this.log("content:" + content);
					news.setContent(content);
				}
			}
			news.setType(News.Type.NEWS);
			this.save(news);
		} catch (Exception ex) {
			this.logException(ex);
		}
		log("Fetch two level end..." + url);
	}

	public void spideIndexPage(String url) {
		log("Fetch news begin..." + url);
		try {
			Parser p = new Parser(url);
			p.setEncoding("UTF-8");
			NodeFilter ul = new TagNameFilter("ul");
			NodeFilter mainNews = new HasAttributeFilter("class", "mainNews");

			NodeList ulList = p.parse(new AndFilter(ul, mainNews));
			for (int i = 0; i < ulList.size(); i++) {
				NodeList liList = ((BulletList) ulList.elementAt(i))
						.getChildren();
				for (Node node : liList.toNodeArray()) {
					log("new:" + node.toPlainTextString());
					if (node.getFirstChild() instanceof LinkTag) {
						LinkTag link = (LinkTag) node.getFirstChild();
						this.spideTwoLevelPage(url + "/" + link.getLink());
					}
				}
			}

		} catch (Exception ex) {
			this.logException(ex);
		}
		log("Fetch news end..." + url);
	}

	public static void main(String[] args) {
		NewsSpider ns = new NewsSpider();
		ns.run();
	}
}
